package org.hackystat.sensorbase.db.dynamodb;

import java.io.IOException;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;

import javax.xml.datatype.XMLGregorianCalendar;

import org.hackystat.sensorbase.db.DbImplementation;
import org.hackystat.sensorbase.resource.projects.jaxb.Project;
import org.hackystat.sensorbase.resource.projects.jaxb.ProjectSummary;
import org.hackystat.sensorbase.resource.projects.jaxb.SensorDataSummaries;
import org.hackystat.sensorbase.resource.projects.jaxb.SensorDataSummary;
import org.hackystat.sensorbase.resource.sensordata.jaxb.SensorData;
import org.hackystat.sensorbase.resource.sensordatatypes.jaxb.SensorDataType;
import org.hackystat.sensorbase.resource.users.jaxb.User;
import org.hackystat.sensorbase.server.Server;
import org.hackystat.utilities.tstamp.Tstamp;
import org.springframework.util.StringUtils;

import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.PropertiesCredentials;
import com.amazonaws.services.dynamodb.AmazonDynamoDBClient;
import com.amazonaws.services.dynamodb.model.*;

public class DynamoDbImplementation extends DbImplementation
{
	private static final String UserTableName = "User";
	private static final String User_KeyAttribute = "Email";
	private static final String User_PasswordAttribute = "Password";
	private static final String User_RoleAttribute = "Role";

	private static final String ProjectTableName = "Project";
	private static final String Project_ProjectNameKey = "Name";
	private static final String Project_OwnerKey = "Owner";
	private static final String Project_StartAttribute = "StartTimeStamp";
	private static final String Project_EndAttribute = "EndTimeStamp";

	private static final String SensorTypeTableName = "SensorDataType";
	private static final String SensorType_NameKey = "Name";

	private static final String SensorDataTableName = "SensorData";
	private static final String SensorData_OwnerKey = "Owner";
	private static final String SensorData_TimestampKey = "Timestamp";
	private static final String SensorData_DataTypeAttribute = "SensorDataType";
	private static final String SensorData_ResourceAttribute = "Resource";
	private static final String SensorData_ToolAttribute = "Tool";
	private static final String SensorData_RuntimeAttribute = "Runtime";

	private static final String AllTables_ModifiedAttribute = "LastModification";
	private static final String AllTables_XmlAttribute = "Xml";
	private static final String AllTables_XmlRefAttribute = "XmlRef";

	private static final String IndexTagStart = "<";
	private static final String IndexSuffix = "Index>";

	private static final Map<String, Boolean> tableNames = new HashMap<String, Boolean>();

	static AmazonDynamoDBClient dynamoDB;
	static List<String> customerTables = new ArrayList<String>();

	static
	{
		tableNames.put(UserTableName, false);
		tableNames.put(ProjectTableName, false);
		tableNames.put(SensorTypeTableName, false);
		tableNames.put(SensorDataTableName, false);
		AWSCredentials credentials;
		try
		{
			credentials = new PropertiesCredentials(
					DynamoDbImplementation.class.getResourceAsStream("AwsCredentials.properties"));
			dynamoDB = new AmazonDynamoDBClient(credentials);
		}
		catch (IOException e)
		{
			throw new RuntimeException("Couldn't load included credentials");
		}
	}

	public DynamoDbImplementation(Server server)
	{
		super(server);
	}

	@Override
	public void initialize()
	{
		boolean createUserTable = true;
		boolean createProjectTable = true;
		boolean createSensorTypeTable = true;
		boolean createSensorDataTable = true;
		ListTablesResult result = dynamoDB.listTables();
		List<String> allTables = result.getTableNames();
		ListTablesRequest request = new ListTablesRequest();
		String continuationTableName = result.getLastEvaluatedTableName();
		while (continuationTableName != null)
		{
			request.setExclusiveStartTableName(continuationTableName);
			result = dynamoDB.listTables(request);
			allTables.addAll(result.getTableNames());
			continuationTableName = result.getLastEvaluatedTableName();
		}

		for (String tableName : allTables)
		{
			if (tableName.equalsIgnoreCase(UserTableName))
			{
				createUserTable = false;
			}
			if (tableName.equalsIgnoreCase(ProjectTableName))
			{
				createProjectTable = false;
			}
			if (tableName.equalsIgnoreCase(SensorTypeTableName))
			{
				createSensorTypeTable = false;
			}
			if (tableName.startsWith(SensorDataTableName))
			{
				createSensorDataTable = false;
			}
		}

		ProvisionedThroughput throughput = new ProvisionedThroughput();
		throughput.setReadCapacityUnits(5L);
		throughput.setWriteCapacityUnits(5L);
		if (createUserTable)
		{
			CreateTableRequest ctr = new CreateTableRequest(UserTableName, new KeySchema(new KeySchemaElement()
					.withAttributeName(User_KeyAttribute).withAttributeType(ScalarAttributeType.S)))
					.withProvisionedThroughput(throughput);
			dynamoDB.createTable(ctr);

			tableNames.put(UserTableName, true);
		}

		if (createProjectTable)
		{
			CreateTableRequest ctr = new CreateTableRequest(ProjectTableName, new KeySchema().withHashKeyElement(
					new KeySchemaElement().withAttributeName(Project_OwnerKey).withAttributeType(ScalarAttributeType.S))
					.withRangeKeyElement(
							new KeySchemaElement().withAttributeName(Project_ProjectNameKey).withAttributeType(
									ScalarAttributeType.S))).withProvisionedThroughput(throughput);
			dynamoDB.createTable(ctr);
			tableNames.put(ProjectTableName, true);
		}

		if (createSensorTypeTable)
		{
			CreateTableRequest ctr = new CreateTableRequest(SensorTypeTableName,
					new KeySchema().withHashKeyElement(new KeySchemaElement().withAttributeName(SensorType_NameKey)
							.withAttributeType(ScalarAttributeType.S))).withProvisionedThroughput(throughput);
			dynamoDB.createTable(ctr);
			tableNames.put(SensorTypeTableName, true);
		}

		if (createSensorDataTable)
		{
			CreateTableRequest ctr = new CreateTableRequest(SensorDataTableName, new KeySchema().withHashKeyElement(
					new KeySchemaElement().withAttributeName(SensorData_OwnerKey).withAttributeType(ScalarAttributeType.S))
					.withRangeKeyElement(
							new KeySchemaElement().withAttributeName(SensorData_TimestampKey).withAttributeType(
									ScalarAttributeType.N))).withProvisionedThroughput(throughput);
			dynamoDB.createTable(ctr);
			tableNames.put(SensorDataTableName, true);
		}
	}

	@Override
	public boolean isFreshlyCreated()
	{
		return tableNames.containsValue(true);
	}

	@Override
	public String getSensorDataIndex()
	{
		return constructIndexXmlFromScan(SensorDataTableName, SensorData_TimestampKey);
	}

	@Override
	public String getSensorDataIndex(User user)
	{
		return constructIndexXmlFromQuery(SensorDataTableName, SensorData_OwnerKey, user.getEmail());
	}

	@Override
	public String getSensorDataIndex(User user, String sdtName)
	{
		Map<String, Condition> constraints = new HashMap<String, Condition>();
		constraints.put(SensorData_OwnerKey, new Condition().withAttributeValueList(new AttributeValue(user.getEmail()))
				.withComparisonOperator(ComparisonOperator.EQ));
		constraints.put(SensorData_DataTypeAttribute, new Condition().withAttributeValueList(new AttributeValue(sdtName))
				.withComparisonOperator(ComparisonOperator.EQ));
		return constructIndexXmlFromScan(SensorDataTableName, SensorData_OwnerKey, constraints);
	}

	@Override
	public String getSensorDataIndex(List<User> users, XMLGregorianCalendar startTime, XMLGregorianCalendar endTime,
			List<String> uriPatterns, String sdt)
	{
		Map<String, Condition> filter = new HashMap<String, Condition>();
		filter.put(
				SensorData_TimestampKey,
				new Condition().withAttributeValueList(new AttributeValue().withN(uselessDateFormatToLong(startTime)),
						new AttributeValue().withN(uselessDateFormatToLong(endTime))).withComparisonOperator(
						ComparisonOperator.BETWEEN));
		if (sdt != null)
		{
			filter.put(SensorData_DataTypeAttribute, new Condition().withAttributeValueList(new AttributeValue(sdt))
					.withComparisonOperator(ComparisonOperator.EQ));
		}

		return constructXmlFor(users, uriPatterns, filter);
	}

	@Override
	public String getSensorDataIndex(List<User> users, XMLGregorianCalendar startTime, XMLGregorianCalendar endTime,
			List<String> uriPatterns, String sdt, String tool)
	{
		Map<String, Condition> filter = new HashMap<String, Condition>();
		filter.put(
				SensorData_TimestampKey,
				new Condition().withAttributeValueList(new AttributeValue().withN(uselessDateFormatToLong(startTime)),
						new AttributeValue().withN(uselessDateFormatToLong(endTime))).withComparisonOperator(
						ComparisonOperator.BETWEEN));
		filter.put(SensorData_DataTypeAttribute, new Condition().withAttributeValueList(new AttributeValue(sdt))
				.withComparisonOperator(ComparisonOperator.EQ));
		filter.put(SensorData_ToolAttribute, new Condition().withAttributeValueList(new AttributeValue(tool))
				.withComparisonOperator(ComparisonOperator.EQ));

		return constructXmlFor(users, uriPatterns, filter);
	}

	@Override
	public String getSensorDataIndex(List<User> users, XMLGregorianCalendar startTime, XMLGregorianCalendar endTime,
			List<String> uriPatterns, int startIndex, int maxInstances)
	{
		Map<String, Condition> filter = new HashMap<String, Condition>();
		filter.put(
				SensorData_TimestampKey,
				new Condition().withAttributeValueList(new AttributeValue().withN(uselessDateFormatToLong(startTime)),
						new AttributeValue().withN(uselessDateFormatToLong(endTime))).withComparisonOperator(
						ComparisonOperator.BETWEEN));

		return constructXmlFor(users, uriPatterns, filter, maxInstances, startIndex);
	}

	@Override
	public String getSensorDataIndexLastMod(User user, XMLGregorianCalendar lastModStartTime,
			XMLGregorianCalendar lastModEndTime)
	{
		Map<String, Condition> filter = new HashMap<String, Condition>();
		filter.put(
				AllTables_ModifiedAttribute,
				new Condition().withAttributeValueList(
						new AttributeValue().withN(uselessDateFormatToLong(lastModStartTime)),
						new AttributeValue().withN(uselessDateFormatToLong(lastModEndTime))).withComparisonOperator(
						ComparisonOperator.BETWEEN));

		return constructIndexXmlFromScan(SensorDataTableName, SensorData_OwnerKey, filter);
	}

	@Override
	public String getProjectSensorDataSnapshot(List<User> users, XMLGregorianCalendar startTime,
			XMLGregorianCalendar endTime, List<String> uriPatterns, String sdt, String tool)
	{
		Map<String, Condition> filter = new HashMap<String, Condition>();
		filter.put(
				SensorData_TimestampKey,
				new Condition().withAttributeValueList(new AttributeValue().withN(uselessDateFormatToLong(startTime)),
						new AttributeValue().withN(uselessDateFormatToLong(endTime))).withComparisonOperator(
						ComparisonOperator.BETWEEN));
		filter.put(SensorData_DataTypeAttribute, new Condition().withAttributeValueList(new AttributeValue(sdt))
				.withComparisonOperator(ComparisonOperator.EQ));
		if (tool != null)
		{
			filter.put(SensorData_ToolAttribute, new Condition().withAttributeValueList(new AttributeValue().withS(tool))
					.withComparisonOperator(ComparisonOperator.EQ));
		}

		List<Map<String, AttributeValue>> result = userAndPatternFilteredScan(users, uriPatterns, filter,
				SensorData_RuntimeAttribute, AllTables_XmlRefAttribute);
		List<Map<String, AttributeValue>> filteredResults = new ArrayList<Map<String, AttributeValue>>();

		long targetRuntime = 0;

		for (Map<String, AttributeValue> value : result)
		{
			long currentRuntime = Long.parseLong(value.get(SensorData_RuntimeAttribute).getN());
			if (currentRuntime > targetRuntime)
			{
				filteredResults.clear();
				targetRuntime = currentRuntime;
			}
			else if (currentRuntime < targetRuntime)
			{
				continue;
			}
			filteredResults.add(value);
		}

		return constructIndexXml(SensorDataTableName, filteredResults);
	}

	@Override
	public String getSensorData(User user, XMLGregorianCalendar timestamp)
	{
		GetItemResult result = dynamoDB.getItem(new GetItemRequest()
				.withTableName(SensorDataTableName)
				.withKey(
						new Key(new AttributeValue().withS(user.getEmail()), new AttributeValue()
								.withN(uselessDateFormatToLong(timestamp)))).withAttributesToGet(AllTables_XmlAttribute));

		return result.getItem().get(AllTables_XmlAttribute).getS();
	}

	@Override
	public boolean hasSensorData(User user, XMLGregorianCalendar timestamp)
	{
		try
		{
			dynamoDB.getItem(new GetItemRequest(SensorDataTableName, new Key(new AttributeValue(user
					.getEmail()), new AttributeValue(uselessDateFormatToLong(timestamp)))).withAttributesToGet(""));
		}
		catch (ResourceNotFoundException x)
		{
			return false;
		}
		return true;
	}

	@Override
	public boolean storeSensorData(SensorData data, String xmlSensorData, String xmlSensorDataRef)
	{
		if (tableNames.get(SensorDataTableName))
		{
			waitForTableReady(SensorDataTableName);
		}

		HashMap<String, AttributeValue> newSensorType = new HashMap<String, AttributeValue>();
		newSensorType.put(SensorData_OwnerKey, new AttributeValue().withS(data.getOwner()));
		newSensorType.put(SensorData_TimestampKey,
				new AttributeValue().withN(uselessDateFormatToLong(data.getTimestamp())));
		newSensorType.put(SensorData_DataTypeAttribute, new AttributeValue().withS(data.getSensorDataType()));
		newSensorType.put(SensorData_ResourceAttribute, new AttributeValue().withS(data.getResource()));
		newSensorType.put(SensorData_RuntimeAttribute,
				new AttributeValue().withN(uselessDateFormatToLong(data.getRuntime())));
		newSensorType.put(SensorData_ToolAttribute, new AttributeValue().withS(data.getTool()));
		addCommonAttributes(newSensorType, xmlSensorData, xmlSensorDataRef);

		dynamoDB.putItem(new PutItemRequest().withTableName(SensorDataTableName).withItem(newSensorType));

		return true;
	}

	@Override
	public void deleteSensorData(User user, XMLGregorianCalendar timestamp)
	{
		dynamoDB.deleteItem(new DeleteItemRequest().withTableName(SensorDataTableName).withKey(
				new Key(new AttributeValue().withS(user.getEmail()), new AttributeValue()
						.withN(uselessDateFormatToLong(timestamp)))));
	}

	@Override
	public void deleteSensorData(User user)
	{
		QueryRequest request = new QueryRequest().withTableName(SensorDataTableName)
				.withAttributesToGet(SensorData_OwnerKey, SensorData_TimestampKey)
				.withHashKeyValue(new AttributeValue(user.getEmail()));
		List<Map<String, AttributeValue>> items = traversePagedQuery(request);

		DeleteItemRequest deleteRequest = new DeleteItemRequest().withTableName(SensorDataTableName);
		for (Map<String, AttributeValue> item : items)
		{
			dynamoDB.deleteItem(deleteRequest.withKey(new Key(item.get(SensorData_OwnerKey), item
					.get(SensorData_TimestampKey))));
		}
	}

	@Override
	public String getSensorDataTypeIndex()
	{
		return constructIndexXmlFromScan(SensorTypeTableName, SensorType_NameKey);
	}

	@Override
	public String getSensorDataType(String sdtName)
	{
		return getEntityXml(SensorTypeTableName, sdtName);
	}

	@Override
	public boolean storeSensorDataType(SensorDataType sdt, String xmlSensorDataType, String xmlSensorDataTypeRef)
	{
		if (tableNames.get(SensorDataTableName))
		{
			waitForTableReady(SensorTypeTableName);
		}

		HashMap<String, AttributeValue> newSensorType = new HashMap<String, AttributeValue>();
		newSensorType.put(SensorType_NameKey, new AttributeValue().withS(sdt.getName()));
		addCommonAttributes(newSensorType, xmlSensorDataType, xmlSensorDataTypeRef);

		dynamoDB.putItem(new PutItemRequest().withTableName(SensorTypeTableName).withItem(newSensorType));

		return true;
	}

	@Override
	public void deleteSensorDataType(String sdtName)
	{
		dynamoDB.deleteItem(new DeleteItemRequest().withTableName(SensorTypeTableName).withKey(
				new Key(new AttributeValue(sdtName))));
	}

	@Override
	public String getUserIndex()
	{
		return constructIndexXmlFromScan(UserTableName, User_KeyAttribute);
	}

	@Override
	public String getUser(String email)
	{
		return getEntityXml(UserTableName, email);
	}

	@Override
	public boolean storeUser(User user, String xmlUser, String xmlUserRef)
	{
		if (tableNames.get(UserTableName))
		{
			waitForTableReady(UserTableName);
		}

		HashMap<String, AttributeValue> newUser = new HashMap<String, AttributeValue>();
		newUser.put(User_KeyAttribute, new AttributeValue().withS(user.getEmail()));
		newUser.put(User_PasswordAttribute, new AttributeValue().withS(user.getPassword()));
		if (user.getRole() != null)
		{
			newUser.put(User_RoleAttribute, new AttributeValue().withS(user.getRole()));
		}
		addCommonAttributes(newUser, xmlUser, xmlUserRef);

		dynamoDB.putItem(new PutItemRequest().withTableName(UserTableName).withItem(newUser));

		return true;
	}

	@Override
	public void deleteUser(String email)
	{
		dynamoDB.deleteItem(new DeleteItemRequest().withTableName(UserTableName).withKey(
				new Key(new AttributeValue(email))));
	}

	@Override
	public String getProjectIndex()
	{
		return constructIndexXmlFromScan(ProjectTableName, Project_ProjectNameKey);
	}

	@Override
	public String getProject(User owner, String projectName)
	{
		GetItemResult result = dynamoDB.getItem(new GetItemRequest().withTableName(ProjectTableName)
				.withKey(new Key(new AttributeValue(owner.getEmail()), new AttributeValue(projectName)))
				.withAttributesToGet(AllTables_XmlAttribute));

		return result.getItem().get(AllTables_XmlAttribute).getS();
	}

	@Override
	public boolean storeProject(Project project, String xmlProject, String xmlProjectRef)
	{
		if (tableNames.get(ProjectTableName))
		{
			waitForTableReady(ProjectTableName);
		}

		HashMap<String, AttributeValue> newProject = new HashMap<String, AttributeValue>();
		newProject.put(Project_OwnerKey, new AttributeValue().withS(project.getOwner()));
		newProject.put(Project_ProjectNameKey, new AttributeValue().withS(project.getName()));
		newProject.put(Project_StartAttribute, new AttributeValue().withS(project.getStartTime().toXMLFormat()));
		newProject.put(Project_EndAttribute, new AttributeValue().withS(project.getEndTime().toXMLFormat()));
		addCommonAttributes(newProject, xmlProject, xmlProjectRef);

		dynamoDB.putItem(new PutItemRequest().withTableName(ProjectTableName).withItem(newProject));
		return true;
	}

	@Override
	public void deleteProject(User owner, String projectName)
	{
		dynamoDB.deleteItem(new DeleteItemRequest().withTableName(ProjectTableName).withKey(
				new Key(new AttributeValue(owner.getEmail()), new AttributeValue(projectName))));
	}

	@Override
	public ProjectSummary getProjectSummary(List<User> users, XMLGregorianCalendar startTime,
			XMLGregorianCalendar endTime, List<String> uriPatterns, String href)
	{
		Map<String, Condition> filter = new HashMap<String, Condition>();
		filter.put(
				SensorData_TimestampKey,
				new Condition().withAttributeValueList(new AttributeValue().withN(uselessDateFormatToLong(startTime)),
						new AttributeValue().withN(uselessDateFormatToLong(endTime))).withComparisonOperator(
						ComparisonOperator.BETWEEN));

		List<Map<String, AttributeValue>> results = userAndPatternFilteredScan(users, uriPatterns, filter,
				SensorData_DataTypeAttribute, SensorData_ToolAttribute);

		Map<String, Map<String, Integer>> sdtInstances = new HashMap<String, Map<String, Integer>>();
		for (Map<String, AttributeValue> value : results)
		{
			String sdt = value.get(SensorData_DataTypeAttribute).getS();
			String tool = value.get(SensorData_ToolAttribute).getS();
			// Don't want null SDTs or Tools, call them the empty string instead.
			if (sdt == null)
			{
				sdt = "";
			}
			if (tool == null)
			{
				tool = "";
			}
			// Now update our numInstance data structure.
			// First, initialize the data structure if this is a new SDT.
			if (!sdtInstances.containsKey(sdt))
			{
				Map<String, Integer> tool2NumInstances = new HashMap<String, Integer>();
				tool2NumInstances.put(tool, 0);
				sdtInstances.put(sdt, tool2NumInstances);
			}
			Map<String, Integer> tool2NumInstances = sdtInstances.get(sdt);
			// Second, initialize the data structure if this is a new tool for a
			// preexisting SDT.
			if (tool2NumInstances.get(tool) == null)
			{
				tool2NumInstances.put(tool, 0);
			}
			// Finally, increment this entry.
			tool2NumInstances.put(tool, tool2NumInstances.get(tool) + 1);
		}
		return makeProjectSummary(href, startTime, endTime, sdtInstances);
	}

	private List<Map<String, AttributeValue>> userAndPatternFilteredScan(List<User> users, List<String> uriPatterns,
			Map<String, Condition> filter, String... attributesToGet)
	{
		UriPatternCollection uris = new UriPatternCollection(uriPatterns);

		List<Map<String, AttributeValue>> results = new ArrayList<Map<String, AttributeValue>>();

		Set<Key> keys = new HashSet<Key>();
		if (uris.IsClientSide)
		{
			for (User user : users)
			{
				filter.put(SensorData_OwnerKey, new Condition().withAttributeValueList(new AttributeValue(user.getEmail()))
						.withComparisonOperator(ComparisonOperator.EQ));
				ScanRequest request = new ScanRequest().withTableName(SensorDataTableName)
						.withAttributesToGet(SensorData_OwnerKey, SensorData_TimestampKey, SensorData_ResourceAttribute)
						.withScanFilter(filter);
				results.addAll(traversePagedScan(request));
			}
			for (Map<String, AttributeValue> item : results)
			{
				if (uris.match(item.get(SensorData_ResourceAttribute)))
				{
					keys.add(new Key().withHashKeyElement(item.get(SensorData_OwnerKey)).withRangeKeyElement(
							item.get(SensorData_TimestampKey)));
				}
			}
		}
		else
		{
			for (UriPattern uriPattern : uris)
			{
				for (User user : users)
				{
					filter.put(SensorData_OwnerKey,
							new Condition().withAttributeValueList(new AttributeValue(user.getEmail()))
									.withComparisonOperator(ComparisonOperator.EQ));

					if (!uriPattern._matchString.equals(""))
					{
						filter.put(SensorData_ResourceAttribute, uriPattern.getNtfsCondition());
						fillKeysFiltered(filter, keys);

						filter.put(SensorData_ResourceAttribute, uriPattern.getUnixCondition());
						fillKeysFiltered(filter, keys);
					}
					else if (uriPattern._type == MatcherType.Unfiltered)
					{
						if (filter.containsKey(SensorData_ResourceAttribute))
						{
							filter.remove(SensorData_ResourceAttribute);
						}
						fillKeysFiltered(filter, keys);
					}
				}
			}
		}

		if (keys.size() > 0)
		{
			Map<String, KeysAndAttributes> target = new HashMap<String, KeysAndAttributes>();
			target.put(SensorDataTableName, new KeysAndAttributes().withAttributesToGet(attributesToGet).withKeys(keys));

			BatchGetItemRequest request = new BatchGetItemRequest().withRequestItems(target);
			BatchGetItemResult result = dynamoDB.batchGetItem(request);
			results = result.getResponses().get(target.keySet().toArray()[0]).getItems();
			while (result.getUnprocessedKeys() != null && result.getUnprocessedKeys().size() > 0)
			{
				Map<String, KeysAndAttributes> remaining = result.getUnprocessedKeys();
				request.setRequestItems(remaining);
				result = dynamoDB.batchGetItem(request);
				results.addAll(result.getResponses().get(target.keySet().toArray()[0]).getItems());
			}
		}
		return results;
	}

	private void fillKeysFiltered(Map<String, Condition> filter, Set<Key> keys)
	{
		for (Map<String, AttributeValue> item : traversePagedScan(new ScanRequest().withTableName(SensorDataTableName)
				.withAttributesToGet(SensorData_OwnerKey, SensorData_TimestampKey).withScanFilter(filter)))
		{
			keys.add(new Key().withHashKeyElement(item.get(SensorData_OwnerKey)).withRangeKeyElement(
					item.get(SensorData_TimestampKey)));
		}
	}

	@Override
	public boolean compressTables()
	{
		return true;
	}

	@Override
	public boolean indexTables()
	{
		return true;
	}

	@Override
	public int getRowCount(String table)
	{
		return dynamoDB.scan(new ScanRequest().withTableName(table).withAttributesToGet().withCount(true)).getCount();
	}

	@Override
	public Set<String> getTableNames()
	{
		return tableNames.keySet();
	}

	private void waitForTableReady(String tableName)
	{
		System.out.println("Waiting for " + tableName + " to become ACTIVE...");

		long endTime = System.currentTimeMillis() + (10 * 60 * 1000);
		while (System.currentTimeMillis() < endTime)
		{
			try
			{
				Thread.sleep(1000 * 20);
			}
			catch (Exception e)
			{
			}
			try
			{
				DescribeTableRequest request = new DescribeTableRequest().withTableName(tableName);
				TableDescription tableDescription = dynamoDB.describeTable(request).getTable();
				String tableStatus = tableDescription.getTableStatus();
				System.out.println("  - current state: " + tableStatus);
				if (tableStatus.equals(TableStatus.ACTIVE.toString()))
				{
					tableNames.put(tableName, false);
					return;
				}
			}
			catch (AmazonServiceException ase)
			{
				if (ase.getErrorCode().equalsIgnoreCase("ResourceNotFoundException") == false)
					throw ase;
			}
		}

		throw new RuntimeException("Table " + tableName + " never went active");
	}

	private void addCommonAttributes(HashMap<String, AttributeValue> item, String xml, String xmlRef)
	{
		item.put(AllTables_XmlAttribute, new AttributeValue().withS(xml));
		item.put(AllTables_XmlRefAttribute, new AttributeValue().withS(xmlRef));
		item.put(AllTables_ModifiedAttribute, new AttributeValue().withN(Long.toString(System.currentTimeMillis())));
	}

	private String getEntityXml(String tableName, String keyValue)
	{
		try
		{
			GetItemResult user = dynamoDB.getItem(new GetItemRequest(tableName, new Key(new AttributeValue(keyValue)))
					.withAttributesToGet(AllTables_XmlAttribute));
			return user.getItem().get(AllTables_XmlAttribute).getS();
		}
		catch (ResourceNotFoundException x)
		{
			return null;
		}
	}

	private String constructIndexXmlFromScan(String tableName, String keyAttributeName)
	{
		return constructIndexXmlFromScan(tableName, keyAttributeName, null);
	}

	private String constructIndexXmlFromScan(String tableName, String keyAttributeName,
			Map<String, Condition> constraints)
	{
		ScanRequest request = new ScanRequest().withTableName(tableName).withAttributesToGet(keyAttributeName,
				AllTables_XmlRefAttribute);
		if (constraints != null)
		{
			request.setScanFilter(constraints);
		}
		List<Map<String, AttributeValue>> items = traversePagedScan(request);

		return constructIndexXml(tableName, items);
	}

	private List<Map<String, AttributeValue>> traversePagedScan(ScanRequest request)
	{
		ScanResult result = dynamoDB.scan(request);
		List<Map<String, AttributeValue>> items = result.getItems();
		while (result.getLastEvaluatedKey() != null)
		{
			request.setExclusiveStartKey(result.getLastEvaluatedKey());
			result = dynamoDB.scan(request);
			items.addAll(result.getItems());
		}
		return items;
	}

	private List<Map<String, AttributeValue>> traversePagedQuery(QueryRequest request)
	{
		QueryResult result = dynamoDB.query(request);
		List<Map<String, AttributeValue>> items = result.getItems();
		while (result.getLastEvaluatedKey() != null)
		{
			request.setExclusiveStartKey(result.getLastEvaluatedKey());
			result = dynamoDB.query(request);
			items.addAll(result.getItems());
		}
		return items;
	}

	private List<Map<String, AttributeValue>> traversePagedBatch(Map<String, KeysAndAttributes> target)
	{
		if (target.size() != 1)
		{
			throw new RuntimeException("I don't support multi-table batches");
		}
		if (target.get(target.keySet().toArray()[0]).getKeys().size() == 0)
		{
			return new ArrayList<Map<String, AttributeValue>>();
		}
		BatchGetItemRequest request = new BatchGetItemRequest().withRequestItems(target);
		BatchGetItemResult result = dynamoDB.batchGetItem(request);
		List<Map<String, AttributeValue>> items = result.getResponses().get(target.keySet().toArray()[0]).getItems();
		while (result.getUnprocessedKeys() != null && result.getUnprocessedKeys().size() > 0)
		{
			Map<String, KeysAndAttributes> remaining = result.getUnprocessedKeys();
			request.setRequestItems(remaining);
			result = dynamoDB.batchGetItem(request);
			items.addAll(result.getResponses().get(target.keySet().toArray()[0]).getItems());
		}
		return items;
	}

	private String constructXmlFor(List<User> users, List<String> uriPatterns, Map<String, Condition> filter)
	{
		return constructXmlFor(users, uriPatterns, filter, 0);
	}

	private String constructXmlFor(List<User> users, List<String> uriPatterns, Map<String, Condition> filter, int limit)
	{
		return constructXmlFor(users, uriPatterns, filter, limit, 0);
	}
	
	private String constructXmlFor(List<User> users, List<String> uriPatterns, Map<String, Condition> filter, int limit, int startIndex)
	{
		List<Map<String, AttributeValue>> results = userAndPatternFilteredScan(users, uriPatterns, filter,
				SensorData_OwnerKey, SensorData_TimestampKey);
		List<Key> keys = new ArrayList<Key>();

		Collections.sort(results, new AttributeValueComparator(SensorData_TimestampKey, ScalarAttributeType.N));
		int toIndex =  startIndex + limit;
		toIndex = toIndex < 1 || toIndex <= startIndex || toIndex > results.size() ? results.size() : toIndex;
		for (Map<String, AttributeValue> item : results.subList(startIndex, toIndex))
		{
			keys.add(new Key().withHashKeyElement(item.get(SensorData_OwnerKey)).withRangeKeyElement(
					item.get(SensorData_TimestampKey)));
		}
		Map<String, KeysAndAttributes> target = new HashMap<String, KeysAndAttributes>();
		target.put(SensorDataTableName,
				new KeysAndAttributes().withAttributesToGet(AllTables_XmlRefAttribute).withKeys(keys));

		return constructIndexXml(SensorDataTableName, traversePagedBatch(target));
	}

	// }

	private String constructIndexXmlFromQuery(String tableName, String keyAttributeName, String keyValue)
	{
		return constructIndexXmlFromQuery(tableName, keyAttributeName, keyValue, null);
	}

	private String constructIndexXmlFromQuery(String tableName, String keyAttributeName, String keyValue,
			Condition constraint)
	{
		QueryRequest request = new QueryRequest().withTableName(tableName).withHashKeyValue(new AttributeValue(keyValue))
				.withAttributesToGet(keyAttributeName, AllTables_XmlRefAttribute);
		if (constraint != null)
		{
			request.setRangeKeyCondition(constraint);
		}
		QueryResult result = dynamoDB.query(request);
		List<Map<String, AttributeValue>> items = result.getItems();
		while (result.getLastEvaluatedKey() != null)
		{
			request.setExclusiveStartKey(result.getLastEvaluatedKey());
			result = dynamoDB.query(request);
			items.addAll(result.getItems());
		}

		return constructIndexXml(tableName, items);
	}

	private String constructIndexXml(String tableName, List<Map<String, AttributeValue>> items)
	{
		StringBuilder builder = new StringBuilder(IndexTagStart);
		builder.append(tableName);
		builder.append(IndexSuffix);
		for (Map<String, AttributeValue> item : items)
		{
			builder.append(item.get(AllTables_XmlRefAttribute).getS());
		}
		builder.append(IndexTagStart);
		builder.append("/");
		builder.append(tableName);
		builder.append(IndexSuffix);

		return builder.toString();
	}

	private String uselessDateFormatToLong(XMLGregorianCalendar calendarValue)
	{
		return Long.toString(calendarValue.toGregorianCalendar().getTimeInMillis());
	}

	/**
	 * Creates a ProjectSummary instances from the passed data.
	 * 
	 * @param href
	 *           The Href representing this resource.
	 * @param startTime
	 *           The startTime for this data.
	 * @param endTime
	 *           The endTime for this data.
	 * @param sdtInstances
	 *           The data structure containing the instances.
	 * @return The ProjectSummary instance.
	 */
	private ProjectSummary makeProjectSummary(String href, XMLGregorianCalendar startTime, XMLGregorianCalendar endTime,
			Map<String, Map<String, Integer>> sdtInstances)
	{
		ProjectSummary projectSummary = new ProjectSummary();
		projectSummary.setHref(href);
		projectSummary.setStartTime(startTime);
		projectSummary.setEndTime(endTime);
		projectSummary.setLastMod(Tstamp.makeTimestamp());
		SensorDataSummaries summaries = new SensorDataSummaries();
		projectSummary.setSensorDataSummaries(summaries);
		int totalInstances = 0;
		for (Map.Entry<String, Map<String, Integer>> entry : sdtInstances.entrySet())
		{
			String sdt = entry.getKey();
			Map<String, Integer> tool2NumInstances = entry.getValue();
			for (Map.Entry<String, Integer> entry2 : tool2NumInstances.entrySet())
			{
				SensorDataSummary summary = new SensorDataSummary();
				summary.setSensorDataType(sdt);
				summary.setTool(entry2.getKey());
				int numInstances = entry2.getValue();
				totalInstances += numInstances;
				summary.setNumInstances(BigInteger.valueOf(numInstances));
				summaries.getSensorDataSummary().add(summary);
			}
		}
		summaries.setNumInstances(BigInteger.valueOf(totalInstances));
		return projectSummary;
	}

	private class UriPattern
	{
		private Condition _condition;

		/**
		 * @return the scan filter condition
		 */
		public Condition getUnixCondition()
		{
			if (_matchString.contains("\\") || _matchString.contains("/"))
			{
				_condition.withAttributeValueList(new AttributeValue(_matchString.replace('\\', '/')));
			}
			else if (_condition.getAttributeValueList() == null || _condition.getAttributeValueList().size() == 0)
			{
				_condition.withAttributeValueList(new AttributeValue().withS(_matchString));
			}
			return _condition;
		}

		public Condition getNtfsCondition()
		{
			if (_matchString.contains("\\") || _matchString.contains("/"))
			{
				_condition.withAttributeValueList(new AttributeValue(_matchString.replace('/', '\\')));
			}
			else if (_condition.getAttributeValueList() == null || _condition.getAttributeValueList().size() == 0)
			{
				_condition.withAttributeValueList(new AttributeValue().withS(_matchString));
			}
			return _condition;
		}

		/**
		 * @return the regex pattern
		 */
		public java.util.regex.Pattern getUnixPattern()
		{
			return Pattern.compile(_matchString.replace("\\\\", "/"));
		}

		public java.util.regex.Pattern getNtfsPattern()
		{
			return Pattern.compile(_matchString.replace("/", "\\\\"));
		}

		private MatcherType _type;

		/**
		 * @return the matcher type, server-side scan api or client side regex
		 */
		public MatcherType Type()
		{
			return _type;
		}

		private String _matchString;

		public UriPattern(String pattern)
		{
			if (pattern.length() == 0)
			{
				_matchString = "";
				_type = MatcherType.NonMatcher;
				return;
			}
			int starsInPattern = StringUtils.countOccurrencesOf(pattern, "*");
			if (pattern.length() > 0 && starsInPattern == pattern.length())
			{
				_matchString = "";
				_type = MatcherType.Unfiltered;
				return;
			}
			if (starsInPattern == 0)
			{
				_matchString = pattern;
				_condition = new Condition().withComparisonOperator(ComparisonOperator.EQ);
				_type = MatcherType.ScanApiCondition;
				return;
			}
			if (starsInPattern == 1 && pattern.endsWith("*"))
			{
				_matchString = pattern.substring(0, pattern.length() - 1);
				_condition = new Condition().withComparisonOperator(ComparisonOperator.BEGINS_WITH);
				_type = MatcherType.ScanApiCondition;
				return;
			}
			if (starsInPattern == 2 && pattern.endsWith("*") && pattern.startsWith("*"))
			{
				_matchString = pattern.substring(1, pattern.length() - 1);
				_condition = new Condition().withComparisonOperator(ComparisonOperator.CONTAINS);
				_type = MatcherType.ScanApiCondition;
				return;
			}
			for (String special : quotable)
			{
				pattern = pattern.replace(special, "\\" + special);
			}
			_matchString = pattern.replace("**", "*").replace("*", ".*");
			_type = MatcherType.RegexPattern;
		}
	}

	public static final String[] quotable = new String[] { "\\", "[", "]", "(", ")", ".", "+", "?", "^", "$" };

	private class UriPatternCollection extends ArrayList<UriPattern>
	{
		/**
		 * 
		 */
		private static final long serialVersionUID = 6993471777639357516L;
		private boolean IsClientSide = false;

		public UriPatternCollection(List<String> patterns)
		{
			for (String pattern : patterns)
			{
				add(new UriPattern(pattern));
			}
		}

		public boolean match(AttributeValue attributeValue)
		{
			StringBuilder regexBuilder = new StringBuilder();
			for (UriPattern pattern : this)
			{
				regexBuilder.append(pattern.getUnixPattern());
				regexBuilder.append('|');
				regexBuilder.append(pattern.getNtfsPattern());
				regexBuilder.append('|');
			}
			regexBuilder.deleteCharAt(regexBuilder.length() - 1);
			return java.util.regex.Pattern.matches(regexBuilder.toString(), attributeValue.getS());
		}

		@Override
		public boolean add(UriPattern e)
		{
			if (e.Type() == MatcherType.RegexPattern)
			{
				IsClientSide = true;
			}
			return super.add(e);
		}
	}

	private class AttributeValueComparator implements Comparator<Map<String, AttributeValue>>
	{
		private final String _attributeName;
		private final ScalarAttributeType _attributeType;

		public AttributeValueComparator(String attributeName, ScalarAttributeType attributeType)
		{
			_attributeName = attributeName;
			_attributeType = attributeType;
		}

		@Override
		public int compare(Map<String, AttributeValue> o1, Map<String, AttributeValue> o2)
		{
			if (o1 == null && o2 == null)
			{
				return 0;
			}
			if (o1 == null)
			{
				return 1;
			}
			if (o2 == null)
			{
				return -1;
			}
			AttributeValue val1 = o1.get(_attributeName);
			AttributeValue val2 = o2.get(_attributeName);
			if (val1 == null && val2 == null)
			{
				return 0;
			}
			if (val1 == null)
			{
				return 1;
			}
			if (val2 == null)
			{
				return -1;
			}
			if (_attributeType == ScalarAttributeType.N)
			{
				Long long1 = Long.parseLong(val1.getN());
				Long long2 = Long.parseLong(val1.getN());
				return long1.compareTo(long2);
			}
			return val1.getS().compareTo(val2.getS());
		}

	}

	public enum MatcherType
	{
		ScanApiCondition, RegexPattern, Literal, Unfiltered, NonMatcher
	}
}
